package crawler;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 超简易爬虫
 * 请求百度，获取关键词结果网页摘要信息
 *
 * Created with IntelliJ IDEA.
 * User: mgm
 * Date: 2019-05-04
 * Time: 15:29
 *
 * 2020年7月4日 百度html返回验证连接
 */
public class SimpleCrawler {

    /** 关键词字符串数组 */
    public static final String[] KEYWORDS = {"IOT"};
    public static final String HOST_BAIDU = "http://www.baidu.com";
    public static final int BAIDU_PN = 10;
    public static final int PAGE = 1;
    public static final String FILE_PATH = "C:/A_HOME/Test";

    public static Pattern patternStart = Pattern.compile("result(-op)* c-container");
    public static Pattern patternTitle = Pattern.compile("<h3 class=\"t(.+?)</h3>");
    public static Pattern patternContent = Pattern.compile("((<div class=\"c-abstract)|(<div class=\"c-span18 c-span-last))(.+?)</div>");
    public static Pattern patternUrl = Pattern.compile("<div class=\"f13\"(.+?)</div>");

    public static void main(String[] args) throws IOException {
        long startTime = System.currentTimeMillis();
        List<Map<String, String>> list = new ArrayList<>(BAIDU_PN * PAGE);
        final List<Map<String, String>> data = Collections.synchronizedList(list);
//        final List<Map<String, String>> data = new ArrayList<>();

        final CountDownLatch countDownLatch = new CountDownLatch(PAGE * KEYWORDS.length);
        ExecutorService executorService = Executors.newFixedThreadPool(8);
        for (final String keyword : KEYWORDS) {
            for (int i = 0; i < PAGE; i++) {
                final int finalI = i;
//            data.addAll(getResData(KEYWORDS[0], finalI));
//                try {
//                    Thread.sleep(5000);
//                } catch (InterruptedException e) {
//                    e.printStackTrace();
//                }
                Thread thread = new Thread(new Runnable() {
                    @Override
                    public void run() {
                        List<Map<String, String>> resData = getResData(keyword, finalI);
                        data.addAll(resData);
                        countDownLatch.countDown();
                        System.out.println("完成：" + keyword+ "-" + finalI +"(" + resData.size()+ "条)");
                    }
                });

                executorService.submit(thread);
                try {
                    Thread.sleep(Math.round(Math.random()*200));
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        }
        executorService.shutdown();


        try {
            countDownLatch.await(5, TimeUnit.MINUTES);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        SimpleDateFormat formatter = new SimpleDateFormat("yyyMMdd");
        File path = new File(FILE_PATH + "/IOT-web-site/" + formatter.format(new Date()));
        if (!path.exists()) {
            path.mkdirs();
        }
        PrintWriter pw = new PrintWriter(new FileWriter(path.getAbsolutePath() + File.separator + KEYWORDS[0]
                + ".csv"), true);
        for (Map<String, String> datum : data) {
            String builderLine = "\"" + datum.get("index") + "\"," +
                    "\"" + datum.get("keyword") + "\"," +
                    "\"" + datum.get("title") + "\"," +
                    "\"" + datum.get("content") + "\"," +
                    "\"" + datum.get("url") + "\"";
            pw.println(builderLine);
        }
        pw.flush();
        pw.close();

        long endTime = System.currentTimeMillis();

        System.out.println("共用时：" + ((endTime - startTime) / 1000.0) + "(s)");

    }

    private static String searchBaidu(String keyword, int pageSize, int pageNumber) {
        String urlStr = HOST_BAIDU + "/s?wd=" + keyword + "&pn=" + (pageSize * pageNumber);
        System.out.println("获取：" + urlStr);
        StringBuilder builder = new StringBuilder();

        try {
            URL url = new URL(urlStr);
            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
            connection.connect();
            BufferedReader reader = new BufferedReader(new InputStreamReader(
                    connection.getInputStream()));
            String line = "";
            while ((line = reader.readLine()) != null) {
                builder.append(line);
            }
            reader.close();
            System.out.println("获取数据完成:" + url.toString());

        } catch (IOException e) {
            e.printStackTrace();
        }


        return builder.toString();
    }

    private static List<Map<String, String>> getResData(String keyword, final int pageNum) {
        String res = searchBaidu(keyword, BAIDU_PN, pageNum);
        return getData(keyword, res, BAIDU_PN * pageNum);
    }

    public static List<Map<String, String>> getData(String keyword, String html, int offsetIndex) {

        int bodyEnd = html.indexOf("</body>");
        List<Map<String, String>> mapList = new ArrayList<>();

        Matcher matcher = patternStart.matcher(html);
        int index = 0;
        while (matcher.find()) {
            Map<String, String> map = new HashMap<>(10);
            int startIndex = matcher.start();
            if (startIndex > bodyEnd) {
                continue;
            }
            Matcher matcherT = patternTitle.matcher(html);
            boolean isFind = matcherT.find(startIndex);
            if (!isFind) {
                continue;
            }
            String title = rvTag(matcherT.group());

            Matcher matcherC = patternContent.matcher(html);
            isFind = matcherC.find(startIndex);
            String content = isFind ? rvTag(matcherC.group()) : "";

            Matcher matcherUrl = patternUrl.matcher(html);
            isFind = matcherUrl.find(startIndex);
            String url = isFind ? rvTag(matcherUrl.group()) : "";

            map.put("index", (offsetIndex + index++) + "");
            map.put("keyword", keyword);
            map.put("title", title);
            map.put("content", content);
            map.put("url", url);
            mapList.add(map);
        }
        return mapList;
    }

    public static String rvTag(String html) {
        String regExHtml = "<[^>]+>";
        String regEx_style = "<style[^>]*?>[\\s\\S]*?<\\/style>"; //定义style的正则表达式

        Pattern p_style = Pattern.compile(regEx_style, Pattern.CASE_INSENSITIVE);
        Matcher m_style = p_style.matcher(html);
        html = m_style.replaceAll(""); //过滤style标签
        Pattern p_html = Pattern.compile(regExHtml, Pattern.CASE_INSENSITIVE);
        Matcher m_html = p_html.matcher(html);
        return m_html.replaceAll("").trim(); //过滤html标签
    }
}
